{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import hashlib\n",
    "from easydict import EasyDict as edict\n",
    "from pathlib import Path\n",
    "\n",
    "import FPR_TPR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FPR-TPR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def autonaming_hash(filepath: Path):\n",
    "    hash = hashlib.md5(filepath.stem.encode(\"utf-8\")).hexdigest()\n",
    "    return hash[:8]\n",
    "\n",
    "def split_rephrased_jsonl_name(filepath: Path):\n",
    "    name = filepath.stem\n",
    "    prefix_idx = name.find(\"@\")\n",
    "    name = name[prefix_idx + 1:]\n",
    "    return name.split(\"__\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RESULT_BASE_DIR = Path(\"../results/Llama-2-13B-GPTQ\")\n",
    "GENERATED_DIR = RESULT_BASE_DIR / \"wm\" / \"SIR\"\n",
    "REPHRASED_DIR = RESULT_BASE_DIR / \"rephrase\" / \"SIR-PRW\"\n",
    "REPHRASED_NOWM_DIR = REPHRASED_DIR.with_stem(REPHRASED_DIR.stem + \"-NOWM\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wm_jsonls = list(filter(lambda p: \"no_wm\" not in p.stem, GENERATED_DIR.glob(\"*.jsonl\")))\n",
    "rep_jsonls = list(REPHRASED_DIR.glob(\"*.jsonl\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for wm_jsonl in wm_jsonls:\n",
    "    nowm_jsonl = wm_jsonl.with_suffix(\".no_wm.jsonl\")\n",
    "    assert nowm_jsonl.exists()\n",
    "    hash_prefix = autonaming_hash(wm_jsonl)\n",
    "    for rep_jsonl in filter(lambda p: hash_prefix in p.stem, rep_jsonls):\n",
    "        rep_nowm_jsonl = REPHRASED_NOWM_DIR / rep_jsonl.name\n",
    "        assert rep_nowm_jsonl.exists()\n",
    "        args = edict({\n",
    "            \"fprs\": [0.01, 0.05, 0.1],\n",
    "            \"gen_wm_jsonl\": wm_jsonl,\n",
    "            \"gen_no_wm_jsonl\": nowm_jsonl,\n",
    "            \"wm_jsonl\": rep_jsonl,\n",
    "            \"no_wm_jsonl\": rep_nowm_jsonl,\n",
    "        })\n",
    "        results = FPR_TPR.main(args)\n",
    "        # visualize\n",
    "        print(\"======================\")\n",
    "        print(\"Experiment: \", REPHRASED_DIR.stem)\n",
    "        print(\"Generator: \", wm_jsonl.stem.split(\"__\")[0])\n",
    "        print(\"Detector: \", wm_jsonl.stem.split(\"__\")[1])\n",
    "        print(\"Hash Prefix: \", hash_prefix)\n",
    "        split_comps = split_rephrased_jsonl_name(rep_jsonl)\n",
    "        print(\"Rephraser: \", split_comps[0])\n",
    "        print(\"Original Detector: \", split_comps[1])\n",
    "        print(\"Rephraser Detector: \", split_comps[2])\n",
    "        print()\n",
    "        for fpr, result in results.items():\n",
    "            gen_tpr, nwr_tpr, old_tpr, new_tpr = (\n",
    "                result[\"generated\"],\n",
    "                result[\"no_wm_rephrased\"],\n",
    "                result[\"old\"],\n",
    "                result[\"new\"],\n",
    "            )\n",
    "            print(f\"FPR: {fpr*100:.2f}%\")\n",
    "            print(f\"  Original TPR: {gen_tpr*100:.2f}%\")\n",
    "            print(f\"  Original TPR after NW rephrasing: {nwr_tpr*100:.2f}%\")\n",
    "            print(f\"  Original TPR after rephrasing: {old_tpr*100:.2f}%\")\n",
    "            print(f\"  Rephrased TPR: {new_tpr*100:.2f}%\")\n",
    "            print()\n",
    "        print(\"======================\")\n",
    "        print()\n",
    "        print()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
